/*
 * 1) sortierte liste von wörtern einlesen
 * 2) wikipedia einlesen und alle wörter mit liste vergleichen
 */
package complet.app.tool.experiment;

import complet.db.DBConn;
import complet.helper.ConfigHelper;
import complet.helper.IOHelper;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.Comparator;
import java.util.TreeMap;

public class experiment3_4 {

    private static BufferedReader in_words = IOHelper.openReadFile("E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\stats\\225_250T\\experiment3\\words_wiki.dat");
    private static BufferedReader wiki = IOHelper.openReadFile("E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\inputfiles\\wikipedia\\wiki.txt");
    private static TreeMap<String, Integer> tm_words = new TreeMap<String, Integer>();
    private static int[] bucket = new int[101];
    private static BufferedWriter out1 = IOHelper.openWriteFile("E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\stats\\225_250T\\experiment3\\buckets_wiki_wiki.dat");

    public static void main(String[] args) {
        ConfigHelper.DB_PATH = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\dbs\\db_ngrams_r5.db";
        ConfigHelper.BUFFER = false;
        DBConn conn = new DBConn();
        try {
            String strLine;
            String line[];
            String word;
            int c = 0;
            while (true && (strLine = in_words.readLine()) != null) {
                line = strLine.split("\t");
                word = line[0].toLowerCase();
                if (!tm_words.containsKey(word) && conn.getWordIndex().get(ConfigHelper.NAME_KEY, word).getSingle() != null) {
                    tm_words.put(line[0].toLowerCase(), c);
                    c++;
                }
            }
        } catch (IOException ex) {
            //IOHelper.logError("(experiment3) Failed to read input file: " + ex.getMessage());
        }

        // Alle Evaluations durchgehen

        try {

            int cnt = 0;

            String strLine;
            String line[];
            String word;
            int c = 0;
            while (true && (strLine = wiki.readLine()) != null) {
                c++;
                if (c % 10000 == 9999) {
                    IOHelper.pln("At " + c + " sentences");
                }
                if (c >= 5000000) {
                    break;
                }

                line = strLine.toLowerCase().split(" ");
                for (int i = 0; i < line.length; i++) {

                    word = line[i].toLowerCase();

                    // Position des Vorschlags
                    if (tm_words.containsKey(word)) {
                        cnt++;
                        int p = (int) ((tm_words.get(word) / (double) (tm_words.size())) * 100);

                        // Count für Bucket erhöhen
                        bucket[p]++;
                    }
                }
            }

            // Ausgeben
            for (int i = 0; i < bucket.length; i++) {
                out1.write(i + "\t " + bucket[i] / (double) (cnt) + "\n");
            }

            out1.flush();
            out1.close();
        } catch (IOException ex) {
            //IOHelper.logError("(experiment3_2) Failed to write result file: " + ex.getMessage());
        }
    }
}
